column_names = df.columns.tolist()
print(column_names)

df1_column_names = df1.columns.tolist()
print(df1_column_names)

def describe_data(data):
    num_observations, num_variables = data.shape

    variable_types = data.dtypes
    print(f"Observations: {num_observations}")
    print(f"Variables: {num_variables}")
    print("\nVariable Storage Display")
    print(f"name\ttype\tformat")

    for variable_name, variable_type in variable_types.items():
        variable_format = ''
        if variable_type == 'int64':
            variable_format = '%10.0g'
        elif variable_type == 'float64':
            variable_format = '%10.0g'
        elif variable_type == 'object':
            variable_format = f'%{max(len(variable_name), 9)}s'

        print(f"{variable_name}\t{variable_type}\t{variable_format}")
        
describe_data(df4)

df8 = df7.copy()
df8['totrev_log'] = np.log1p(df8['totrev'])
df8['totexp_log'] = np.log1p(df8['totexp'])

import matplotlib.pyplot as plt

plt.figure(figsize=(10, 5))
plt.scatter(df8['black_hisp_share'], df8['totrev_log'], alpha=0.5)
plt.title('Correlation between Black/Hispanic Enrollment Share and Log of Total Revenues')
plt.xlabel('Black/Hispanic Enrollment Share')
plt.ylabel('Log of Total Revenues')
plt.grid(True)
plt.show()

plt.figure(figsize=(10, 5))
plt.scatter(df8['black_hisp_share'], df8['totexp_log'], alpha=0.5)
plt.title('Correlation between Black/Hispanic Enrollment Share and Log of Total Expenditures')
plt.xlabel('Black/Hispanic Enrollment Share')
plt.ylabel('Log of Total Expenditures')
plt.grid(True)
plt.show()

# Question 4 

df9=df8.copy()
df9['quantile'] = pd.qcut(df9['black_hisp_share'], q=100, labels=False)

quantile_stats = df9.groupby('quantile').agg({
    'totrev_log': 'mean',
    'totexp_log': 'mean'
}).reset_index()

import matplotlib.pyplot as plt
quantile_stats_array = quantile_stats.to_numpy()

plt.figure(figsize=(10, 5))
plt.plot(quantile_stats_array[:, 0], quantile_stats_array[:, 1], label='Avg Log Total Per Pupil Revenue')
plt.plot(quantile_stats_array[:, 0], quantile_stats_array[:, 2], label='Avg Log Total Per Pupil Expenditures')
plt.title('Average Log Total Per Pupil Revenue and Expenditures by Black/Hispanic Student Quantiles')
plt.xlabel('Quantiles of Black/Hispanic Student Share')
plt.ylabel('Average Log Value')
plt.legend()
plt.grid(True)
plt.show()

# Group the data by state (stabbr)
state_groups = df9.groupby('stabbr')

# Calculate the average Black/Hispanic share and per-pupil expenditures for each state
state_summary = state_groups.agg({
    'black': 'mean',
    'hisp': 'mean',
    'black_hisp_share':'mean',
    'pupil_per_exp': 'mean'
}).reset_index()

state_summary.head()

# Rename the columns if needed
state_summary.rename(columns={'black_hisp_share': 'Avg_black_hist_Share', 'pupil_per_exp': 'Avg_Per_Pupil_Exp'}, inplace=True)

# Plot the scatter plot with the x-axis as the combined avg Black/Hispanic share and the y-axis as the average per-pupil expenditures
plt.scatter(state_summary['Avg_black_hist_Share'], state_summary['Avg_Per_Pupil_Exp'], alpha=0.5)

# Add labels with two-letter state abbreviations
for i, txt in enumerate(state_summary['stabbr']):
    plt.annotate(txt, (state_summary['Avg_black_hist_Share'].iloc[i], state_summary['Avg_Per_Pupil_Exp'].iloc[i]))

# Add axis labels and a title
plt.xlabel('Average Black/Hispanic Enrollment Share')
plt.ylabel('Average Per-Pupil Expenditures')
plt.title('Relationship between State-Average Student Expenditures and Combined Black/Hispanic Enrollment Share')

# Display the plot
plt.grid(True)  

gdf.head(1)

# basic plot
gdf.plot()

# Simple map for US States based on the downloaded file
f,ax = plt.subplots(1,1, figsize=(8,6), sharex=True, sharey=True, dpi=300)
plt.title('Simple Map of US States')
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="3%",pad=0,alpha=0.5)
us49.plot('ALAND', ax=ax, alpha=0.5, cmap='Pastel1', edgecolor='k', legend=True, cax=cax, linewidth=0.1)
plt.show()

# Starting combine the data we have processed with the States data with geo information
# Checks for state_summary(done in previous questions)
state_summary.head(3)

data_types = my_data.dtypes
print(data_types)

# Remove leading/trailing whitespaces and convert to uppercase
us49['STUSPS'] = us49['STUSPS'].str.strip().str.upper()
my_data['stabbr'] = my_data['stabbr'].str.strip().str.upper()

# Perform an inner join
df_join = us49.merge(my_data, left_on='STUSPS', right_on='stabbr', how='inner')
df_join.head(3)

import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable

def StatesPlot(df, data, cmap):
    fig, ax = plt.subplots(1, 1, figsize=(15, 10), sharex=True, sharey=True, dpi=300)
    fig.tight_layout()
    plt.title('United States Map - Variable = ' + data)
    ax.set_axis_off()
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("right", size="3%", pad=0.5, alpha=0.5)
    
    # Plot the data using the specified colormap
    df.plot(data, ax=ax, alpha=0.5, cmap=cmap, edgecolor='k', legend=True, cax=cax, linewidth=0.1)
    
    # Customize the colorbar label and appearance
    cax.set_title(data, fontsize=12)
    
    plt.show()

# Example usage with df_join:
StatesPlot(df_join, 'Avg_black_hist_Share', 'YlOrRd')

# Specify the name that is used to seach for the data
import geopandas as gpd
import matplotlib.pyplot as plt

shapefile_path = 'Green_Cart_Bnd_2008_DOHMH_2010/Green_Cart_Bnd_2008_DOHMH_2010.shp'

gdf_states_ny = gpd.read_file(shapefile_path)

# gdf_states_ny.head()
gdf_states_ny.plot()
plt.title('Map of Your Dataset')
plt.xlabel('Longitude')
plt.ylabel('Latitude')
plt.show()

gdf_states_ny.head()

# Map of NYC
f, ax = plt.subplots(1, 1, figsize=(12, 10), sharex=True, sharey=True, dpi=300)
gdf_states_ny.plot('FIRST_BO_1', ax=ax, alpha=0.5, cmap='Pastel1', edgecolor='k', legend=True, linewidth=0.1)
plt.title('Map of NYC')

divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="3%", pad=0.1)

gdf_states_ny.plot('FIRST_BO_1', ax=ax, alpha=0.5, cmap='Pastel1', edgecolor='k', cax=cax, linewidth=0.1)
plt.show()

import geopandas as gpd
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import make_axes_locatable

shapefile_path = 'tl_2022_49_tabblock20/tl_2022_49_tabblock20.shp'
gdf_states_ut = gpd.read_file(shapefile_path)

f, ax = plt.subplots(1, 1, figsize=(12, 10), sharex=True, sharey=True, dpi=300)
gdf_states_ut.plot('ALAND20', ax=ax, alpha=0.5, cmap='Pastel1', edgecolor='k', legend=True, linewidth=0.1)
plt.title('Map of Utah')

divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="3%", pad=0.1)

gdf_states_ut.plot('ALAND20', ax=ax, alpha=0.5, cmap='Pastel1', edgecolor='k', cax=cax, linewidth=0.1)
plt.show()

# Specify the name that is used to seach for the data
place_name = "Salt Lake City, Utah, USA"

# Get place boundary related to the place name as a geodataframe
area = ox.geocode_to_gdf(place_name)

# Check the data type
area

# List key-value pairs for tags
tags = {'building': True}   

buildings = ox.geometries_from_place(place_name, tags)
buildings.head()

# Plot footprints 
buildings.plot()